import requests
from xjlibrary.our_file_dir import BaseDir

curPath = BaseDir.get_file_dir_absolute(__file__)
TopPath = BaseDir.get_upper_dir(curPath, -2)
sPath = BaseDir.get_new_path(TopPath, "download", "rucjournalold", "download", "listpage")
BaseDir.create_dir(sPath)

def httpRequest(pagesize):
    headers = {
        'Referer': 'http://ipub.exuezhe.com/search.html?adv=1#anchorList',
        'Content-Type': 'application/x-www-form-urlencoded',
        'X-Requested-With': 'XMLHttpRequest',
        'Accept': '*/*',
        'Accept-Language': 'zh-CN',
        'Accept-Encoding': 'gzip, deflate',
        'User-Agent': 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)',
        'Host': 'ipub.exuezhe.com',
        'Content-Length': '449',
        'DNT': '1',
        'Proxy-Connection': 'Keep-Alive',
        'Pragma': 'no-cache',
        'Cookie': 'Hm_lvt_478f1d5b61e997f79060eb076697d8f1=1459494651,1459495870,1459496513; zlzx=0ottwe2x5a3hge4xh5wauecg; Hm_lpvt_478f1d5b61e997f79060eb076697d8f1=1459496599; zlzx_customid=15988433; zlzx_custom_name=%e6%b5%99%e6%b1%9f%e5%a4%a7%e5%ad%a6'
    }
    postData = {
        'data': '{"TreeCode":"","DbCode":1,"StartYear":"1995","EndYear":"2017","ListSearchCondition":[{"Type":"0","KeyWord":"","IsAccurate":"2","RelationShip":1},{"Type":"0","KeyWord":"","IsAccurate":"2","RelationShip":"1"}],"Order":1,"PageSize":20,"PageIndex":%s,"IsBuy":0}' % str(
            pagesize)}
    url = 'http://ipub.exuezhe.com/Qw/Search'
    try:
        r = requests.post(url, data=postData, headers=headers, timeout=60)
        return r.text
    except Exception as e:
        return "Error:" + str(e)


def main():
    i = 1
    while i <= 29590:
        htmlText = httpRequest(i)
        htmlText = htmlText.replace("\n", "").replace("\r", "").replace("\0", "") + "\n"
        BaseDir.single_add_file(BaseDir.get_new_path(sPath, "result.big_json"), htmlText)
        print(str(i) + " page Write success!")
        i = i + 1


if __name__ == "__main__":
    main()
